import hanlp
from hanlp.utils.lang.en.english_tokenizer import tokenize_english


tokenizer = hanlp.load('CTB6_CONVSEG')
res = tokenizer("工信处女干事")
print(res)

tokenizer = tokenize_english
res = tokenizer('not the destination, but the scenery along the should be and the')
print(res)
